﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace CopySniffer.Core.Tokenizers
{
    public class RegexTokenizer : ITokenizer
    {
        protected readonly string regexPattern;
        protected const string DefaultRegexPattern = @"((?<word>\w+)[\ \,\""\(\)\']*)+";
            //@"((?<line>.+)(?:(?<newline>\r\n)|$))+";

        public RegexTokenizer()
            : this(DefaultRegexPattern)
        {
        }

        protected RegexTokenizer(string regexPattern)
        {
            this.regexPattern = regexPattern;
        }

        protected virtual string StripString(string str)
        {
            StringBuilder sb = new StringBuilder(str);
            //sb.Replace(Environment.NewLine, " ");
            int count;
            do
            {
                count = sb.Length;
                sb.Replace("  ", " ");
                sb.Replace('\t', ' ');
            }
            while (sb.Length != count);
            return sb.ToString();
        }

        Dictionary<string, string> replacements = new Dictionary<string,string>();

        #region ITokenizer Members

        public IEnumerable<string> GetTokens(string source)
        {
            source = StripString(source);
            Regex regex = new Regex(regexPattern);

            var q = from Match m in regex.Matches(source)
                    select m.Value.Replace(Environment.NewLine, " ");

            return q.ToList();
        }

        public IEnumerable<string> GetTokens(string source, int maxCount)
        {
            var result = GetTokens(source);
            Random rnd = new Random();
            result = result.OrderByDescending(e => e.Length * ( 1 + rnd.NextDouble())).Take(maxCount);
            return result;
        }

        #endregion
    }
}
